# model

# responsible for the RAID-5 file-upload (performance). an advantage of this raid-level is, that the data is split before uploaded in the storage-services,
# so no storage-provider or no storage-service has the complete data (security-aspect).
#===================================================================================================================================================================================
# IMPORTANT-LINKS:
# http://code.google.com/intl/de-DE/appengine/docs/python/blobstore/blobreaderclass.html
#===================================================================================================================================================================================

from google.appengine.ext import webapp
from google.appengine.ext.webapp import blobstore_handlers

from boto.connection import HTTPRequest
from boto.s3.resumable_download_handler import *
from boto.s3.connection import *
from boto.s3.key import *

from Utilities import *

from controller.BuildStorageInfo import *

from model.octopus.datastore.OctopusCloudDatabaseOpenStack import OctopusCloudDatabaseOpenStack
from model.octopus.datastore.OctopusCloudDatabaseRaidFive import OctopusCloudDatabaseRaidFive

from google.appengine.ext.db import GqlQuery
from google.appengine.ext.blobstore import BlobInfo

from google.appengine.ext.blobstore.blobstore import BlobReader
from google.appengine.ext.blobstore.blobstore import *

import types


class FileUploadHandlerRaidFive(blobstore_handlers.BlobstoreUploadHandler): 
    def get(self, pSelf, uploadedFiles, key, acl):
        utilities = Utilities()
        
        username = utilities.getUsername()
        userText = utilities.getUserText()
        
        amazon = utilities.getAmazonText()
        google = utilities.getGoogleText()
        openStack = utilities.getOpenStackText()
        
        if username:
            urlText = utilities.getUrlTextLogout()
            url = utilities.getLogout(pSelf)
            
            bucketname = utilities.getBucketname(username)
            
            # get the available credentials and storage-amount
            credentials = utilities.getCredentials(username)
            credentialAmount = credentials.count()
            
            if uploadedFiles:
                # TODO: 'NOTE'
                # check if the necessary attributes (key-name, acl, content-type) were set respectively chosen
                # if not, do nothing and print a hint-message for the user
                # if it is, than do the procedure below...
                
                
                # determine the file-size and maybe some other file-dependent information (blobFile-attribute is a blobInfo-object)
                blobFile = uploadedFiles[0]

                # get the blob-file-key (necessary to identify the blob-file)
                blobKey = blobFile.key()
                
                # get the blob-file-size
                blobSize = blobFile.size
                
                # get the blob-file-name (the same like the key-name)
                blobName = blobFile.filename
                
                # get the blob-file content-type
                blobContentType = blobFile.content_type
                
                # get the blob-file time-stamp
                blobTimestamp = blobFile.creation
                
                # get the md5-checksum of the original file
                blobFileMD5 = blobFile.md5_hash
                
                # index to identify the divisor
                divisor = credentialAmount -1
                
                # calculate the size of the first-file-part
                blobSizeOne = (blobSize / divisor)

                # calculate the size of the second-file-part (is mostly smaller than the first part) 
                blobSizeTwo = blobSize - blobSizeOne

                # get the first file-part as a blob-reader-object                
                blobReaderOne = blobstore.BlobReader(blobKey, buffer_size=blobSizeOne, position=0)
                
                # set the read-position
                blobReaderOne.seek(0)
                
                # read the first part of the file (in byte)
                blobReaderPartOne = blobReaderOne.read(blobSizeOne)
                blobReaderPartOneSize = len(blobReaderPartOne)
                
                # make a file for the first part
                blobFilePartOne = blobReaderOne.blob_info
                blobFilePartOneSize = blobFilePartOne.size
                
                # get the second file-part as a blob-reader-object
                blobReaderTwo = blobstore.BlobReader(blobKey, buffer_size=blobSizeTwo, position=blobSizeOne)
                
                # set the read-position
                blobReaderTwo.seek(blobSizeOne)
                
                # read the second part of the file (in bytes)
                blobReaderPartTwo = blobReaderTwo.read(blobSizeTwo)
                blobReaderPartTwoSize = len(blobReaderPartTwo)
                
                # make a file for the second part
                blobFilePartTwo = blobReaderTwo.blob_info
                blobFilePartTwoSize = blobFilePartTwo.size
                
                # convert the first blob-reader-file form string into integer
                blobReaderPartOneInteger = self.convertBlobReaderStringToInteger(utilities, blobReaderPartOne)                  
                blobReaderPartOneLength = len(blobReaderPartOneInteger)
                
                # convert the second blob-reader-file from string into integer
                blobReaderPartTwoInteger = self.convertBlobReaderStringToInteger(utilities, blobReaderPartTwo)
                blobReaderPartTwoLength = len(blobReaderPartTwoInteger)

                # if the second file part has less bits than the first one, add the necessary amount to the end..
                blobReaderPartOneLength = self.adjustBlobReaderLength(utilities, blobReaderPartOneInteger, blobReaderPartTwoInteger)
                
                # compute the parity of the two file parts (xor-operation)
                blobReaderParityList = self.computeParity(utilities, blobReaderPartOneInteger, blobReaderPartTwoInteger)
                    
                # convert the parity form integer into string
                parity = self.convertParityIntegerToString(utilities, blobReaderParityList)
                paritySize = len(parity)
                
                # convert the parity-string into a blob-reader-object
                blobReaderParity = blobstore.BlobReader(parity)
                
                # make a file from the parity-string
                parityFile = StringIO.StringIO(parity)
                
                # get the storage-order
                storageOrderList = utilities.getStorageOrderList(username)
                
                if credentials:
                    storageOne = ''
                    storageTwo = ''
                    storageThree = ''
                    
                    fileOrderOne = 1
                    fileOrderTwo = 2
                    fileOrderThree = 3
                    
                    for i in range(len(storageOrderList)):
                        storageOne = storageOrderList[i].storageOne
                        storageTwo = storageOrderList[i].storageTwo
                        storageThree = storageOrderList[i].storageThree
                    
                    # check the storage-service for the first-storage-entry
                    if storageOne == amazon:
                        amazonStorageConnection = utilities.getStorageServiceConnection(username, amazon)
                        amazonStorageConnection = amazonStorageConnection[0]
                        self.uploadFile(utilities, amazon, amazonStorageConnection, bucketname, blobReaderPartOne, key, blobContentType, blobTimestamp, acl, blobReaderPartOneLength, blobFileMD5)
                        
                        nKey = utilities.createNewKey(amazonStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, blobReaderPartOne)
                        
                        self.persistInfoForFilePartOne(username, blobName, blobSize, blobFileMD5, md5, fileOrderOne, amazon)
                    elif storageOne == google:
                        googleStorageConnection = utilities.getStorageServiceConnection(username, google)
                        googleStorageConnection = googleStorageConnection[0]
                        self.uploadFile(utilities, google, googleStorageConnection, bucketname, blobReaderPartOne, key, blobContentType, blobTimestamp, acl, blobReaderPartOneLength, blobFileMD5)

                        nKey = utilities.createNewKey(googleStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, blobReaderPartOne)
                        
                        self.persistInfoForFilePartOne(username, blobName, blobSize, blobFileMD5, md5, fileOrderOne, google)
                    elif storageOne == openStack:
                        openStackStorageConnection = utilities.getStorageServiceConnection(username, openStack)
                        openStackStorageConnection = openStackStorageConnection[0]
                        self.uploadFile(utilities, openStack, openStackStorageConnection, bucketname, blobReaderPartOne, key, blobContentType, blobTimestamp, acl, blobReaderPartOneLength, blobFileMD5)

                        nKey = utilities.createNewKey(openStackStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, blobReaderPartOne)
                        
                        self.persistInfoForFilePartOne(username, blobName, blobSize, blobFileMD5, md5, fileOrderOne, openStack)
                    
                    # check the storage-service for the second-storage-entry
                    if storageTwo == amazon:
                        amazonStorageConnection = utilities.getStorageServiceConnection(username, amazon)
                        amazonStorageConnection = amazonStorageConnection[0]
                        self.uploadFile(utilities, amazon, amazonStorageConnection, bucketname, blobReaderPartTwo, key, blobContentType, blobTimestamp, acl, blobReaderPartTwoLength, blobFileMD5)

                        nKey = utilities.createNewKey(amazonStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, blobReaderPartTwo)
                        
                        self.persistInfoForFilePartTwo(username, blobName, blobSize, blobFileMD5, md5, fileOrderTwo, amazon)
                    elif storageTwo == google:
                        googleStorageConnection = utilities.getStorageServiceConnection(username, google)
                        googleStorageConnection = googleStorageConnection[0]
                        self.uploadFile(utilities, google, googleStorageConnection, bucketname, blobReaderPartTwo, key, blobContentType, blobTimestamp, acl, blobReaderPartTwoLength, blobFileMD5)

                        nKey = utilities.createNewKey(googleStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, blobReaderPartTwo)
                        
                        self.persistInfoForFilePartTwo(username, blobName, blobSize, blobFileMD5, md5, fileOrderTwo, google)
                    elif storageTwo == openStack:
                        openStackStorageConnection = utilities.getStorageServiceConnection(username, openStack)
                        openStackStorageConnection = openStackStorageConnection[0]
                        self.uploadFile(utilities, openStack, openStackStorageConnection, bucketname, blobReaderPartTwo, key, blobContentType, blobTimestamp, acl, blobReaderPartTwoLength, blobFileMD5)

                        nKey = utilities.createNewKey(openStackStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, blobReaderPartTwo)
                        
                        self.persistInfoForFilePartTwo(username, blobName, blobSize, blobFileMD5, md5, fileOrderTwo, openStack)
                    
                    # check the storage-service for the third-storage-entry
                    if storageThree == amazon:
                        amazonStorageConnection = utilities.getStorageServiceConnection(username, amazon)
                        amazonStorageConnection = amazonStorageConnection[0]
                        self.uploadFile(utilities, amazon, amazonStorageConnection, bucketname, parity, key, blobContentType, blobTimestamp, acl, paritySize, blobFileMD5)

                        nKey = utilities.createNewKey(amazonStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, parity)
                        
                        self.persistInfoForFilePartThree(username, blobName, blobSize, blobFileMD5, md5, fileOrderThree, amazon)
                    elif storageThree == google:
                        googleStorageConnection = utilities.getStorageServiceConnection(username, google)
                        googleStorageConnection = googleStorageConnection[0]
                        self.uploadFile(utilities, google, googleStorageConnection, bucketname, parity, key, blobContentType, blobTimestamp, acl, paritySize, blobFileMD5)

                        nKey = utilities.createNewKey(googleStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, parity)
                        
                        self.persistInfoForFilePartThree(username, blobName, blobSize, blobFileMD5, md5, fileOrderThree, google)
                    elif storageThree == openStack:
                        openStackStorageConnection = utilities.getStorageServiceConnection(username, openStack)
                        openStackStorageConnection = openStackStorageConnection[0]
                        self.uploadFile(utilities, openStack, openStackStorageConnection, bucketname, parity, key, blobContentType, blobTimestamp, acl, paritySize, blobFileMD5)

                        nKey = utilities.createNewKey(googleStorageConnection, bucketname, key)
                        md5 = self.determineMD5ForBlobReader(utilities, nKey, parity)
                        
                        self.persistInfoForFilePartThree(username, blobName, blobSize, blobFileMD5, md5, fileOrderThree, openStack)
                else:
                    keyList = self.getKeyList(utilities, username, bucketname)                
                    
#                    hintText = _('An error occurred - could not get the available credentials')
                    hintText = 'An error occurred - could not get the available credentials'
                    hintColor = 'red'
                    
                    self.callBuilder(userText, username, url, urlText, hintText, hintColor, keyList)
                    
                # combine the file-part-information and persist them in the local database (OctopusCloudDatabaseRaidFive)
                self.perpareRaidFiveInfo(utilities, username, blobName, blobSize, blobFileMD5, fileOrderOne, fileOrderTwo, fileOrderThree, blobKey, blobTimestamp)

                # resort the storage-order
                utilities.resortStorageOrder(username)
                
                # when the upload is done - redirect to the select-data-view
                pSelf.redirect('/selectdata')                        
            else:
                keyList = self.getKeyList(utilities, username, bucketname)
    
#                hintText = _('No file chosen - please choose a file')
                hintText = 'No file chosen - please choose a file'
                hintColor = 'red'
                
                self.callBuilder(userText, username, url, urlText, hintText, hintColor, keyList)
        else:
            pSelf.redirect('/')
    
    # determine the md5-checksum for a blob-reader-file-part
    def determineMD5ForBlobReader(self, utilities, k, blobReaderFile):
        md5 = utilities.computeMD5FromString(k, blobReaderFile)
        return md5

    # persist some info for the first file-part
    def persistInfoForFilePartOne(self, username, blobName, blobSize, blobFileMD5, md5, fileOrderOne, storageOne):
        raidFiveInfo = OctopusCloudDatabaseRaidFive(
                                                    user = username,
                                                    filename = blobName,
                                                    filesize = blobSize,
                                                    md5File = blobFileMD5,
                                                    md5PartOne = md5,
                                                    fileOrderOne = fileOrderOne,
                                                    storageOne = storageOne
                                                    )
        raidFiveInfo.put()
        
    # persist some info for the second file-part
    def persistInfoForFilePartTwo(self, username, blobName, blobSize, blobFileMD5, md5, fileOrderTwo, storageTwo):
        raidFiveInfo = OctopusCloudDatabaseRaidFive(
                                                    user = username,
                                                    filename = blobName,
                                                    filesize = blobSize,
                                                    md5File = blobFileMD5,
                                                    md5PartTwo = md5,
                                                    fileOrderTwo = fileOrderTwo,
                                                    storageTwo = storageTwo
                                                    )
        raidFiveInfo.save()
    
    # persist some info for the third file-part
    def persistInfoForFilePartThree(self, username, blobName, blobSize, blobFileMD5, md5, fileOrderThree, storageThree):
        raidFiveInfo = OctopusCloudDatabaseRaidFive(
                                                    user = username,
                                                    filename = blobName,
                                                    filesize = blobSize,
                                                    md5File = blobFileMD5,
                                                    md5Parity = md5,
                                                    fileOrderThree = fileOrderThree,
                                                    storageThree = storageThree
                                                    )
        raidFiveInfo.save()
    
    # prepare some information to persist them in the local database
    def perpareRaidFiveInfo(self, utilities, username, blobName, blobSize, blobFileMD5, orderOne, orderTwo, orderThree, blobKey, blobLastMod):
        raidFiveInfoStorageOneEntry = utilities.getRaidFiveInfoStorageOneEntry(username, blobName, blobFileMD5, orderOne)
        resultsStorageOne = raidFiveInfoStorageOneEntry.fetch(100)
        
        raidFiveInfoStorageTwoEntry = utilities.getRaidFiveInfoStorageTwoEntry(username, blobName, blobFileMD5, orderTwo)
        resultsStorageTwo = raidFiveInfoStorageTwoEntry.fetch(100)
        
        raidFiveInfoStorageThreeEntry = utilities.getRaidFiveInfoStorageThreeEntry(username, blobName, blobFileMD5, orderThree)
        resultsStorageThree = raidFiveInfoStorageThreeEntry.fetch(100)
        
        # md5-values
        md5PartOne = None
        md5PartTwo = None
        md5Parity = None
        
        # file-order
        fileOrderOne = None
        fileOrderTwo = None
        fileOrderThree = None
        
        # storage-order
        storageOne = None
        storageTwo = None
        storageThree = None
        
        for resultStorageOne in resultsStorageOne:
            md5PartOne = resultStorageOne.md5PartOne
            fileOrderOne = resultStorageOne.fileOrderOne
            storageOne = resultStorageOne.storageOne
            
        for resultStorageTwo in resultsStorageTwo:
            md5PartTwo = resultStorageTwo.md5PartTwo
            fileOrderTwo = resultStorageTwo.fileOrderTwo
            storageTwo = resultStorageTwo.storageTwo
    
        for resultStorageThree in resultsStorageThree:
            md5Parity = resultStorageThree.md5Parity
            fileOrderThree = resultStorageThree.fileOrderThree
            storageThree = resultStorageThree.storageThree
        
        utilities.deleteRaidFiveInfoEntry(resultsStorageOne)
        utilities.deleteRaidFiveInfoEntry(resultsStorageTwo)
        utilities.deleteRaidFiveInfoEntry(resultsStorageThree)
        
        self.persistRaidFiveInfo(username, blobName, blobSize, blobFileMD5, md5PartOne, md5PartTwo, md5Parity, fileOrderOne, fileOrderTwo, fileOrderThree, storageOne, storageTwo, storageThree, blobKey, blobLastMod)
    
    # persist raid-5-information in the local database
    def persistRaidFiveInfo(self, username, blobName, blobSize, blobFileMD5, md5PartOne, md5PartTwo, md5Parity, fileOrderOne, fileOrderTwo, fileOrderThree, storageOne, storageTwo, storageThree, blobKey, blobLastMod):
        raidFiveInfo = OctopusCloudDatabaseRaidFive(
                                                    user = username,
                                                    filename = blobName,
                                                    filesize = blobSize,
                                                    md5File = blobFileMD5,
                                                    md5PartOne = md5PartOne,
                                                    fileOrderOne = fileOrderOne,
                                                    storageOne = storageOne,
                                                    md5PartTwo = md5PartTwo,
                                                    fileOrderTwo = fileOrderTwo,
                                                    storageTwo = storageTwo,
                                                    md5Parity = md5Parity,
                                                    fileOrderThree = fileOrderThree,
                                                    storageThree = storageThree,
                                                    blobInfoKey = str(blobKey),
                                                    lastModified = blobLastMod
                                                    )
        raidFiveInfo.put()
    
    # converts the string-representation into a integer-representation - needed for the parity-calculation...
    def convertBlobReaderStringToInteger(self, utilities, blobReader):
        blobReaderInteger = utilities.convertStringToInt(blobReader)
        return blobReaderInteger
        
    # adjust the length of a blob-reader-part, so that the parts has the same length
    def adjustBlobReaderLength(self, utilities, blobReaderPartOneInteger, blobReaderPartTwoInteger):
        blobReaderPartOneLength = utilities.adjustStringFileLength(blobReaderPartOneInteger, blobReaderPartTwoInteger)
        return blobReaderPartOneLength
    
    # compute the parity of the two file-parts (xor)
    def computeParity(self, utilities, blobReaderPartOneInteger, blobReaderPartTwoInteger):
        blobReaderParityList = utilities.performXorOperation(blobReaderPartOneInteger, blobReaderPartTwoInteger)
        return blobReaderParityList
        
    # converts the integer-representation of the parity-calculation into a normal string    
    def convertParityIntegerToString(self, utilities, blobReaderParityList):
        parity = utilities.convertIntToString(blobReaderParityList)
        return parity
    
    # handle the file-upload..
    def uploadFile(self, utilities, regionname, storageConnection, bucketname, blobReaderFile, key, blobContentType, lastMod, acl, size, blobFileMD5):
        utilities.uploadFileRaidFive(regionname, storageConnection, bucketname, blobReaderFile, key, blobContentType, lastMod, acl, size, blobFileMD5)
    
    # loads a list of keys that will be shown in the responsible view (by default: amazon)
    def getKeyList(self, utilities, username, bucketname):
        storageConnection = self.getAmazonStorageConnection(utilities, username)
        keyList = utilities.getBucketKeyList(storageConnection, bucketname)
        return keyList
        
    # responsible for the amazon-storage-connection
    def getAmazonStorageConnection(self, utilities, username):
        credentials = utilities.getCredentials(username)
        results = credentials.fetch(100)
        
        if credentials:
            for result in results:
                if result.regionname == utilities.getAmazonText():
                    amazonAccessKey = result.accessKey
                    amazonSecretAccessKey = utilities.decodeSecretAccessKey(username, result.secretAccessKey)
                    amazonEndpointURL = utilities.getAmazonURL()
                    
                    amazonStorageConnection = utilities.getS3Connection(amazonAccessKey, amazonSecretAccessKey, amazonEndpointURL)
                    
                    return amazonStorageConnection
        
    # responsible for the call of the get-method of the BuildStorageInfo-Class
    def callBuilder(self, userText, username, url, urlText, hintText, hintColor, keyList):
        templateValues = {
                          'userText': userText,
                          'user': username,
                          'urlText': urlText,
                          'url': url,
                          'urlText': urlText,
                          'hintText': hintText,
                          'hintColor': hintColor
                          }
        
        builder = BuildStorageInfo()
        builder.get(self, templateValues, keyList)

            
